{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "step 100, entropy loss: 0.213628, l2_loss: 624.899719 ,total_loss:0.238624\n",
      "0.9366\n",
      "step 200, entropy loss: 0.214766, l2_loss: 832.865112 ,total_loss:0.248081\n",
      "0.9333\n",
      "step 300, entropy loss: 0.147119, l2_loss: 1003.615540 ,total_loss:0.187264\n",
      "0.9511\n",
      "step 400, entropy loss: 0.093882, l2_loss: 1181.871948 ,total_loss:0.141157\n",
      "0.9526\n",
      "step 500, entropy loss: 0.101052, l2_loss: 1333.489258 ,total_loss:0.154391\n",
      "0.9608\n",
      "step 600, entropy loss: 0.060714, l2_loss: 1386.400513 ,total_loss:0.116170\n",
      "0.9668\n",
      "step 700, entropy loss: 0.163616, l2_loss: 1329.817261 ,total_loss:0.216809\n",
      "0.97\n",
      "step 800, entropy loss: 0.038910, l2_loss: 1278.688721 ,total_loss:0.090057\n",
      "0.9723\n",
      "step 900, entropy loss: 0.030472, l2_loss: 1255.915039 ,total_loss:0.080709\n",
      "0.9741\n",
      "step 1000, entropy loss: 0.148712, l2_loss: 1240.908203 ,total_loss:0.198348\n",
      "0.9731\n",
      "step 1100, entropy loss: 0.084976, l2_loss: 1250.924316 ,total_loss:0.135013\n",
      "0.9717\n",
      "step 1200, entropy loss: 0.113980, l2_loss: 1222.591064 ,total_loss:0.162884\n",
      "0.9712\n",
      "step 1300, entropy loss: 0.064236, l2_loss: 1183.186523 ,total_loss:0.111563\n",
      "0.9754\n",
      "step 1400, entropy loss: 0.086798, l2_loss: 1149.377441 ,total_loss:0.132774\n",
      "0.9782\n",
      "step 1500, entropy loss: 0.029716, l2_loss: 1116.709229 ,total_loss:0.074385\n",
      "0.9776\n",
      "step 1600, entropy loss: 0.054061, l2_loss: 1087.808228 ,total_loss:0.097574\n",
      "0.9773\n",
      "step 1700, entropy loss: 0.039553, l2_loss: 1066.394165 ,total_loss:0.082209\n",
      "0.9796\n",
      "step 1800, entropy loss: 0.021871, l2_loss: 1029.531860 ,total_loss:0.063053\n",
      "0.9806\n",
      "step 1900, entropy loss: 0.093617, l2_loss: 1006.808228 ,total_loss:0.133890\n",
      "0.9827\n",
      "step 2000, entropy loss: 0.013361, l2_loss: 985.827881 ,total_loss:0.052794\n",
      "0.9811\n",
      "step 2100, entropy loss: 0.009000, l2_loss: 966.879028 ,total_loss:0.047675\n",
      "0.9816\n",
      "step 2200, entropy loss: 0.042912, l2_loss: 948.107300 ,total_loss:0.080836\n",
      "0.9813\n",
      "step 2300, entropy loss: 0.013852, l2_loss: 923.558960 ,total_loss:0.050794\n",
      "0.9824\n",
      "step 2400, entropy loss: 0.010163, l2_loss: 903.061707 ,total_loss:0.046285\n",
      "0.9809\n",
      "step 2500, entropy loss: 0.005170, l2_loss: 890.693176 ,total_loss:0.040797\n",
      "0.9832\n",
      "step 2600, entropy loss: 0.010133, l2_loss: 877.828735 ,total_loss:0.045246\n",
      "0.9828\n",
      "step 2700, entropy loss: 0.006651, l2_loss: 865.642761 ,total_loss:0.041276\n",
      "0.9826\n",
      "step 2800, entropy loss: 0.013731, l2_loss: 853.334106 ,total_loss:0.047864\n",
      "0.9826\n",
      "step 2900, entropy loss: 0.017455, l2_loss: 839.180359 ,total_loss:0.051022\n",
      "0.983\n",
      "step 3000, entropy loss: 0.007350, l2_loss: 824.830750 ,total_loss:0.040344\n",
      "0.9831\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "def swish(x):\n",
    "  return x*tf.nn.sigmoid(x)\n",
    "\n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0,x ,alpha*tf.nn.elu(x))\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "def tanh(x):\n",
    "    return tf.nn.tanh(x)\n",
    "def prelu(x):\n",
    "    a=0.2\n",
    "    return  tf.where(x>0,x,a*x)\n",
    "#swish函数在所有激活函数中效果好些\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "#  return relu(x)\n",
    "#  return tf.nn.digmoid(x)\n",
    "#  return tf.nn.elu(x)\n",
    "   return swish(x)\n",
    "#  return prelu(x)\n",
    "#  return tanh(x)\n",
    "def initialize(shape,stddev=0.1):\n",
    "  return tf.truncated_normal(shape,stddev=stddev)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "\n",
    "\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "#一层隐层个数\n",
    "#尝试增加隐层个数可以提高准确率\n",
    "L1_units_count = 300\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "\n",
    "\n",
    "#exponetial lr decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000,tf.shape(x)[0]))\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch #//2\n",
    "current_learning_rate = tf.multiply(init_learning_rate,\n",
    "                                    tf.pow(0.575,tf.to_float(decay_times)))\n",
    "\n",
    "\n",
    "W_1 = tf.Variable(initialize([784,L1_units_count],\n",
    "                            stddev=np.sqrt(2/784)))\n",
    "b_1 = tf.Variable(tf.constant(0.001,shape=[L1_units_count]))\n",
    "logits_1 = tf.matmul(x,W_1) + b_1\n",
    "output_1 = activation(logits_1)\n",
    "\n",
    "#输出层\n",
    "L2_units_count = 10\n",
    "W_2 = tf.Variable(initialize([L1_units_count,\n",
    "                              L2_units_count],\n",
    "                            stddev = np.sqrt(2/L1_units_count)))\n",
    "b_2 = tf.Variable(tf.constant(0.001,shape=[L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1,W_2) + b_2\n",
    "#不加激活\n",
    "y = logits_2\n",
    "\n",
    "\n",
    "#Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None,10])\n",
    "\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "#softmax_cross_entropy_with_logits内部会调用softmax，直接使用logits，不要经过激活函数\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "#l2 正则 对w优化过程约束，使之成为凸优化\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)\n",
    "total_loss = cross_entropy + 4e-5*l2_loss\n",
    "#添加adam\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss,global_step=global_step)\n",
    "\n",
    "\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "\n",
    "#Train \n",
    "\n",
    "for step in range(3000):\n",
    "  batch_xs,batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 1e-2\n",
    "  _, loss,l2_loss_value,total_loss_value,current_lr_value = \\\n",
    "      sess.run(\n",
    "               [train_step,cross_entropy,l2_loss,total_loss,\n",
    "                current_learning_rate],\n",
    "               feed_dict={x: batch_xs, y_:batch_ys,\n",
    "                          init_learning_rate:lr})\n",
    "    \n",
    "  if (step+1) % 100== 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f ,total_loss:%f'%\n",
    "               (step+1, loss, l2_loss_value,total_loss_value))\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
